/**
 * 爬虫获取 Gitee 仓库的贡献者列表
 */
const request = require('request')
const cheerio = require('cheerio');

function Crawler() {
}

/**
 * 从 Gitee 贡献者列表页面抓取贡献者头像信息
 * @param url 贡献者列表页
 * @param callback 回调函数
 * @param retry 重试次数。当抓取失败时会重试
 * @param arr 初始列表。当存在分页时，该数组中存放之前获取到的所有图片链接
 */
Crawler.grab = function (url, callback, retry, arr) {
    const opts = {
        url: url,
        headers: {
            'User-Agent': `Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.115 Safari/537.36`,
            'Cookie': 'oschina_new_user=false; close_wechat_tour=true; remote_way=ssh; tz=Etc%2FGMT-8; user_locale=zh-CN; gitee_user=true; Hm_lvt_24f17767262929947cc3631f99bfd274=1616156131,1616158097,1616405494,1616421077; sensorsdata2015jssdkcross=%7B%22distinct_id%22%3A%22784199%22%2C%22first_id%22%3A%22176be187721562-04d6747fc48276-c791039-2073600-176be187722807%22%2C%22props%22%3A%7B%22%24latest_traffic_source_type%22%3A%22%E7%9B%B4%E6%8E%A5%E6%B5%81%E9%87%8F%22%2C%22%24latest_search_keyword%22%3A%22%E6%9C%AA%E5%8F%96%E5%88%B0%E5%80%BC_%E7%9B%B4%E6%8E%A5%E6%89%93%E5%BC%80%22%2C%22%24latest_referrer%22%3A%22%22%2C%22%24latest_utm_source%22%3A%22badge%22%2C%22%24latest_utm_medium%22%3A%22badge%22%2C%22%24latest_utm_campaign%22%3A%22pr-badge%22%7D%2C%22%24device_id%22%3A%22176be187721562-04d6747fc48276-c791039-2073600-176be187722807%22%7D; access_token=f043e6268a23c20d3cf86d2928a994cf; Hm_lpvt_24f17767262929947cc3631f99bfd274=1616464088; gitee-session-n=VW5rTnY4NGdZVXd3TlRBRHVvd0cvdEkrMnIweEE5OUJlU3BWZ3U3Y2dZWmZvRkE2OThUT2xvZHVwYkJvK0Uya2xmMW1MNStLR2FoTXI1UW91OTZlZG9MQ2VQVk5SYU5RdFBxSkxjeHd6NGVYVnM5OTJEZXFiMDlxa2pPYjRtQ0FtaWVYUnIyMllsU0hJYTNWMzNabWg5MjdzT3NwNWliTGJaOG1CbTdtMytnc3FTOGtpb0VhcWR6VHpMVW1yRFkwUklnUEZITGZJUlBBT3A0TjlsN3N2M3Fmd1lLdUh0U3YyMnQxVVFnaXRhRWp6THBpQWZlT1ZkeEt3TWtVNFVaM2llYURnaGZKaDZOcHU5ZjBzYUgybXdOejF1eGFXSndNeXNYTzFyQ1JWWk9aYWM5dFc2aG5JeG9DTXQ2L0lNeG5aajhKdWRmWFc3Wm4zejZJUTI1NVhwSzY1YVhTYUIxVE5ONVRBRG16STVFPS0tN2thMEptUEFrdXZoVXRlNUVSZEpKZz09--61e120413cc7d4b2185135ceb3cc7cdb81d864f6',
        }
    }
    const that = this
    request(opts, (err, req) => {
        if (err) {
            console.log("failed", err)
            callback && callback(arr)
            return;
        }
        const json = parseAvatars(req.body.toString());
        let avatars = json.data;
        if ((!avatars || avatars.length === 0 || !avatars[0]) && retry > 0) {
            console.log("没获取到有效的列表，重试一次")
            that.grab(url, callback, retry - 1, arr)
        } else {
            avatars = (arr || []).concat(avatars || [])
            if (json.hasNext) {
                that.grab(json.nextPageUrl, callback, 1, avatars)
                return;
            }
            callback && callback(avatars)
        }
    })
}

/**
 * 解析头像
 * @param html
 * @returns {{nextPageUrl: (string|null), data: [], hasNext: boolean}}
 */
function parseAvatars(html) {
    const $ = cheerio.load(html);
    const contributors = $('.user-list-item');
    const data = [];

    contributors.map(function (node) {
        const contributor = $(this);
        let avatar = contributor.find('a:eq(0) img').attr("src");
        if (!avatar) {
            avatar = contributor.find('.content .header a').text();
        }
        data.push(avatar);
    });
    const nextPage = $("#git-discover-page a[rel=next].icon")
    const hasNext = nextPage !== null && nextPage.length > 0
    return {
        data: data,
        hasNext: hasNext,
        nextPageUrl: hasNext ? 'https://gitee.com' + nextPage.attr("href") : null
    };
}

module.exports = Crawler;
